package words.tokenizer;

import java.util.ArrayList;
import java.util.List;

import stemmer.Stemmer;

public class WordsTokenizer {
	
	private Stemmer _stemmer;
	private ArrayList<String> _lastWords;

	public WordsTokenizer(Stemmer stemmer){
		_stemmer = stemmer;
		_lastWords = new ArrayList<String>();
	}

	public List<String> tokenize(String text) {
		_lastWords.clear();
		List<String> words = new ArrayList<String>();
		String[] wordsAsArray = text.split(" ");
		for (int i=0; i<wordsAsArray.length; i++) {
			String each = wordsAsArray[i];
			each = each.toLowerCase();
			if(!each.isEmpty()){
				char firstChar = each.charAt(0);
				if((firstChar>='a'&&firstChar<='z')) {
					each = _stemmer.baseFormOf(each);
					_lastWords.add(each);
					words.add(each);
				}
			}
		}
		return words;
	}

	public List<String> nGramsOfLastText(int nGrams, boolean lastFirst) {
		List<String> nGramsList = new ArrayList<String>();
		for(int i=0; i<_lastWords.size(); i++){
			if((i>nGrams-2)&&nGrams>1){
				StringBuffer buffer = new StringBuffer();				
				for (int j = i-(nGrams-1); j <=i; j++) {
					buffer.append(_lastWords.get(j));
					if(!nGramsList.contains(buffer.toString())) nGramsList.add(buffer.toString());
					if(j!=i) buffer.append("-");
				}
			}
			if(lastFirst){
				String firstLastGram = _lastWords.get(0)+"_"+_lastWords.get(_lastWords.size()-1);
				nGramsList.add(firstLastGram);				
			}
		}
		return nGramsList;
	}

}
